#!/usr/bin/env python3

# Copyright (C) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0

# -*- Python -*-

"""! P4C driver
Takes care of creating output structure and guiding the whole compilation.
"""

import argparse
import json
import os
import os.path
import re
import sys
import time

import p4c_src.bfn_version as p4c_version
from p4c_src.driver import BackendDriver
from p4c_src.util import find_bin, find_file


def parse_version(value):
    match = re.search(r"^(\d+)\.(\d+)\.(\d+)$", value)
    if not match:
        raise ValueError("Invalid version: '{}'".format(value))
    return tuple(map(int, match.groups()))


class CompilationError(Exception):
    """! Raised when a P4 program fails to compile"""

    pass


def checkEnv():
    """!
    @return the top source directory, or None if can not determine it.
    """
    top_src_dir = None
    if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
        top_src_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..')
        scripts_dir = os.path.join(top_src_dir, 'scripts')
        if not os.path.isdir(scripts_dir):
            # we are building out of the source directory, so we need to get the source dir
            # from the cmake cache
            cache_file = os.path.join(os.environ['P4C_BIN_DIR'], '..', 'CMakeCache.txt')
            if not os.path.isfile(cache_file):
                # This is a funny setup, where we can't find our configuration, so we should
                # not run anything depending on scripts
                return None
            with open(cache_file) as cmake_cache:
                src_dir_pattern = re.compile(r'BFN_P4C_SOURCE_DIR:STATIC=(.*)$')
                for line in cmake_cache:
                    res = src_dir_pattern.match(line)
                    if res:
                        top_src_dir = res.group(1)
                        break
        if os.path.isdir(top_src_dir):
            return top_src_dir

    return None


class BarefootBackend(BackendDriver):
    """!
    Customized version of public driver to specify our options,
    how to work with them and process them.
    """

    def __init__(self, target, arch, argParser):
        """!
        @param target Target device
        @param arch Target architecture
        @param argParser argparse instace to setup arguments
        """
        BackendDriver.__init__(self, target, arch, argParser)
        self.compilation_time = 0.0
        self.conf_file = None
        self.contexts = {}  # generated by the assembler
        self.mau_json = {}  # remove when the compiler adds the manifest
        self.metrics = {}
        self.program_name = None

        # commands
        self.add_command('preclean-dirs', 'find')
        self.add_command('preclean-files', 'find')
        self.add_command('preclean-runtime', 'rm')
        self.add_command('preprocessor', 'cc')
        self.add_command('compiler', os.path.join(os.environ['P4C_BIN_DIR'], 'p4c-barefoot'))
        self.add_command('cleaner', 'rm')

        self.runVerifiers = False
        top_src_dir = checkEnv()
        if top_src_dir:
            scripts_dir = os.path.join(top_src_dir, 'scripts')
            if os.path.isdir(scripts_dir):
                self.add_command(
                    'manifest-verifier', os.path.join(scripts_dir, 'validate_manifest')
                )
                self._commandsEnabled.append('manifest-verifier')
                self.add_command('verifier', os.path.join(scripts_dir, 'validate_output.sh'))
                self.runVerifiers = True

        # order of commands
        self.enable_commands(
            [
                'preclean-dirs',
                'preclean-files',
                'preclean-runtime',
                'preprocessor',
                'compiler',
                'assembler',
                'summary_logging',
                'p4c-gen-conf',
                'cleaner',
            ]
        )

        # additional options
        self.add_command_line_options()

    def add_command_line_options(self):
        """! Sets up argparser instance to contain all needed arguments."""
        # BackendDriver.add_command_line_options(self)
        self._argGroup = self._argParser.add_argument_group("Barefoot Networks specific options")
        self._argGroup.add_argument(
            "--create-graphs",
            help="Create parse and table flow graphs",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--display-power-budget",
            help="Display MAU power summary after compilation.",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--no-link",
            dest="skip_linker",
            help="Run up to linker",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "-s",
            dest="run_post_compiler",
            help="Only run assembler",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--archive",
            nargs='?',
            help="Archive all outputs into a single tar.bz2 file.\n"
            + "Note: it can not be the argument before source file"
            + " without specifying the archive name!",
            const="__default__",
            default=None,
        )
        self._argGroup.add_argument(
            "--archive-source",
            action="store_true",
            default=False,
            help="Add source outputs to the archive.",
        )
        self._argGroup.add_argument(
            "--bf-rt-schema",
            action="store",
            default=None,
            help="Generate and write BF-RT JSON schema  to the specified file",
        )
        self._argGroup.add_argument(
            "--no-bf-rt-schema",
            action="store_true",
            default=False,
            help="Do not generate the BF-RT JSON schema",
        )
        self._argGroup.add_argument(
            "--backward-compatible",
            action="store_true",
            default=False,
            help="Set compiler to be backward compatible with p4c-tofino",
        )
        self._argGroup.add_argument(
            "--skip-compilation",
            action="store",
            help="Skip compiling pipes whose name contains one of the" "'pipeX' substring",
        )
        self._argGroup.add_argument(
            "--skip-precleaner",
            action="store_true",
            default=False,
            dest="skip_precleaner",
            help="Skips precleaning phase which deletes all directory"
            " content before compilation.",
        )
        self._argGroup.add_argument(
            "--auto-init-metadata",
            action="store_true",
            default=False,
            help="Automatically initialize metadata to false or 0. This "
            "is always enabled for P4_14. Initialization of individual "
            "fields can be disabled by using the pa_no_init annotation.",
        )
        self._argGroup.add_argument(
            "--disable-egress-latency-padding",
            action="store_true",
            help="Disables adding match"
            " dependent stages to the egress pipeline to "
            " achieve minimum required latency",
        )

        self._argGroup.add_argument(
            "--parser-timing-reports",
            help="Generate parser timing reports",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--parser-bandwidth-opt",
            help="Perform parser bandwidth optimization",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--egress-intrinsic-metadata-opt",
            help="Optimize unused egress intrinsic metadata",
            action="store_true",
            default=False,
        )

        self._argGroup.add_argument(
            "--ir-to-json",
            default=None,
            help="Dump the IR after midend to JSON in the specified file.",
        )
        self._argGroup.add_argument(
            "--verbose",
            action="store",
            default=0,
            type=int,
            choices=[0, 1, 2, 3],
            help="Set compiler logging verbosity level: 0=OFF, 1=SUMMARY, 2=INFO, 3=DEBUG",
        )
        self._argGroup.add_argument(
            "--Wdisable",
            action="append",
            nargs="?",
            default=None,
            const="",
            type=str,
            help="Disable a compiler diagnostic, or disable all warnings "
            "if no diagnostic is specified.",
        )
        self._argGroup.add_argument(
            "--Werror",
            action="append",
            nargs="?",
            default=None,
            const="",
            type=str,
            help="Report an error for a compiler diagnostic, or treat all "
            "warnings as errors if no diagnostic is specified.",
        )
        self._argGroup.add_argument(
            "--help-warnings",
            help="Print warning types that can be used for --Werror and --Wdisable options.",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--p4runtime-force-std-externs",
            action="store_true",
            default=False,
            help="Generate P4Info file using standard extern messages"
            " instead of Tofino-specific ones, for a P4 program written"
            " for a Tofino-specific arch",
        )
        self._argGroup.add_argument(
            "--no-dead-code-elimination", action="store_true", default=False, help=argparse.SUPPRESS
        )  # Do not use dead code elimination.
        self._argGroup.add_argument(
            "--placement",
            action="store",
            type=str,
            default=None,
            choices=["pragma"],
            help=argparse.SUPPRESS,
        )  # Ignore all dependencies placement
        self._argGroup.add_argument(
            "--log-hashes",
            action="store_true",
            default=False,
            help="Log hash functions in use to mau.hashes.log.",
        )
        self._argGroup.add_argument(
            "--quick-phv-alloc",
            action="store_true",
            default=False,
            help="Reduce PHV allocation search space for faster compilation.",
        )
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            for debugger in ["gdb", "cgdb", "lldb"]:
                self._argGroup.add_argument(
                    "--" + debugger,
                    action="store_true",
                    default=False,
                    help="run the backend compiler under the %s debugger" % (debugger,),
                )
            self._argGroup.add_argument(
                "--validate-output",
                action="store_true",
                default=False,
                help="run context.json validation",
            )
            self._argGroup.add_argument(
                "--validate-manifest",
                action="store_true",
                default=False,
                help="run manifest validation",
            )
        self._argGroup.add_argument(
            "--schema-versions",
            help="Print all used schema versions",
            action="store_true",
            default=False,
        )
        self._argGroup.add_argument(
            "--num-stages-override",
            help="Override default number of available MAU stages",
            action="store",
            default=0,
            type=int,
        )
        self._argGroup.add_argument(
            "--program-name",
            help="Program name overriding the default name derived from source file name.",
            action="store",
            default=None,
            type=str,
            dest="program_name",
            required=False,
        )

    def configPrecleaner(self, opts, output_dir):
        """! Configures precleaners command to remove possible previous
        build artifacts.
        Use directory and file delete lists since PTF usually uses the output
        directory for its files and this helps the user not delete their
        files if for some reason . is used as output directory.

        @param opts Options object
        @param output_dir Output directory (will be cleaned)
        """
        # List of directory patterns to delete in the output folder
        # Folder graphs and logs might not be located within pipe folder in
        # case of P4_14 code.
        dir_delete_list = ["pipe*", "graphs", "logs", "visualization"]
        # List of file patterns to delete in the output folder.
        file_delete_list = ["*.json", "*.conf", "*.p4pp", "*.bfa", "*.log", "*.bin", "*.txt"]
        # Pipe might have different name and there can be multiple of them
        # the names can be extracted from manifest.json
        manifest_path = os.path.join(self._output_directory, 'manifest.json')
        if os.path.isfile(manifest_path):
            manifest = dict()
            with open(manifest_path, "r") as manifest_json:
                try:
                    manifest = json.load(manifest_json)
                except json.decoder.JSONDecodeError:
                    pass  # Json was not correct
            if "programs" in manifest:
                for program in manifest["programs"]:
                    if "pipes" in program:
                        for pipe in program["pipes"]:
                            # Find all pipe names and add it to delete list
                            if "pipe_name" in pipe:
                                dir_delete_list.append(pipe["pipe_name"])
        # Delete previous build directories
        self.add_command_option("preclean-dirs", output_dir + " \\(")
        first = True
        for d in dir_delete_list:
            if not first:
                self.add_command_option("preclean-dirs", "-or")
            self.add_command_option("preclean-dirs", "-name " + d)
            first = False
        self.add_command_option("preclean-dirs", "\\) -type d -exec rm -rf {} +")
        # Delete all files in output directory created by previous builds
        self.add_command_option("preclean-files", output_dir + " \\(")
        first = True
        for f in file_delete_list:
            if not first:
                self.add_command_option("preclean-files", "-or")
            self.add_command_option("preclean-files", "-path " + f)
            first = False
        self.add_command_option("preclean-files", "\\) -type f -exec rm -f {} +")
        # Delete --p4runtime-files if existent on new path
        if opts.p4runtime_files is not None:
            self.add_command_option("preclean-runtime", "-f " + opts.p4runtime_files)
        else:
            self.disable_commands(["preclean-runtime"])

    def config_preprocessor(self, targetDefine):
        """! Configures preprocessor option.
        @param targetDefine Define name (what goes after -D)
        """
        self.add_command_option('preprocessor', "-E -x assembler-with-cpp")
        self.add_command_option('preprocessor', "-D" + targetDefine)
        self.add_command_option('preprocessor', p4c_version.macro_defs)

    def config_compiler(self, targetDefine):
        """! Configures compile option.
        @param targetDefine Define name (what goes after -D)
        """
        self.add_command_option('compiler', "--nocpp")
        self.add_command_option('compiler', "-D" + targetDefine)
        self.add_command_option('compiler', p4c_version.macro_defs)

    def config_assembler(self, targetName):
        """! Configures assembler option.
        @param targetName Target name
        """
        self._targetName = targetName
        self._no_link = False
        self._multi_parsers = False

    def config_p4c_gen_conf(self, maxPipe):
        self.add_command_option('p4c-gen-conf', "--max-pipe " + str(maxPipe))

    def config_warning_modifiers(self, arguments, option):
        """! Behaviour of warnings emitted by p4c can be modified by two options:
        --Werror which turns all/selected warnings into errors
        --Wdisable which ignores all/selected warnings
        Both accept either no further options or they accept comma separated list of strings
        or they can occur multiple times with a different CSL each time. If argparser is properly configured
        (action="append", nargs="?", default=None, const="", type=str) it will create a list of strings
        (plain or CSLs) or empty string (if no further option was provided).
        You can then pass parsed argument to this function to properly select between everything or something
        and to properly parse CSLs.

        @param arguments Warning arguments
        @param option String value, either "disable" or "error"
        """
        if option != "disable" and option != "error":
            raise Exception(
                "Programmer error - config_warning_modifiers does not support option " + option
            )

        all = len(arguments) == 1 and arguments[0] == ""

        if all:
            self.add_command_option('compiler', '--W{}'.format(option))
        else:
            for diag in arguments:
                subdiags = diag.split(',')
                for sd in subdiags:
                    self.add_command_option('compiler', '--W{}={}'.format(option, sd))

    def should_not_check_input(self, opts):
        """!
        @return True if input should not be checked, otherwise False
        """
        return opts.help_pragmas or opts.help_warnings

    def process_command_line_options(self, opts):
        """! Main parsing or command line options
        @param opts Object holding set arguments
        """
        # Add assembler options.
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            bfas = find_file('.', 'bfas')
        else:
            bfas = find_file(os.environ['P4C_BIN_DIR'], 'bfas')

        bfrt_schema = find_file(os.environ['P4C_BIN_DIR'], 'bfrt_schema.py')
        p4c_gen_conf = find_file(os.environ['P4C_BIN_DIR'], 'p4c-gen-conf')
        self.add_command('assembler', bfas)
        self.add_command('bf-rt-verifier', bfrt_schema)
        self.add_command('p4c-gen-conf', p4c_gen_conf)
        self._commandsEnabled.append('assembler')

        BackendDriver.process_command_line_options(self, opts)

        # P4 program name is by default derived from the source file name,
        # or it can be explicitly specified by command line option.
        self.program_name = (
            self._source_basename if opts.program_name is None else opts.program_name
        )

        # Set some defaults:
        # --archive implies -g (debug_info)
        # -g implies --verbose 1 and --create-graphs
        if opts.archive is not None:
            if not opts.debug_info:
                self.add_command_option('compiler', '-g')
            opts.debug_info = True
        if opts.verbose > 0 and not opts.debug_info:
            self.add_command_option('compiler', '-g')
        if opts.debug_info and opts.verbose == 0:
            opts.verbose = 1
        if opts.debug_info:
            opts.create_graphs = True

        # Enable the verbose mode if it is passed via the command line
        # the self._verbose variable controls the verbosity mode inside
        # the BackendDriver class. Verbose mode is enabled when debug mode
        # is detected.
        self.debug_info = opts.debug_info
        self.verbose = opts.verbose
        if opts.verbose == 3:
            # Enable more verbose backend driver mode
            self._verbose = True

        self.checkVersionTargetArch(opts.target, opts.language, opts.arch)
        self.language = opts.language

        # Make sure we don't have conflicting debugger options.
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            if sum(int(x) for x in [opts.gdb, opts.cgdb, opts.lldb]) > 1:
                self.exitWithError("Cannot use more than one debugger at a time.")

        # process the options related to source file
        if self._output_directory == '.':
            # if no output directory set, set it to <program_name.target>
            self._output_directory = "{}.{}".format(self.program_name, self._target)
        output_dir = self._output_directory
        basepath = "{}/{}".format(output_dir, self.program_name)

        # Make precleaner delete artifacts from previous builds if
        # enabled and there is a folder to cleanup.
        # It deletes everything in the output folder
        if not opts.skip_precleaner and os.path.isdir(output_dir):
            self.configPrecleaner(opts, output_dir)
        else:
            self.disable_commands(['preclean-files', 'preclean-dirs', 'preclean-runtime'])

        if not opts.run_preprocessor_only:
            self.add_command_option('preprocessor', "-o")
            self.add_command_option('preprocessor', "{}.p4pp".format(basepath))
        self.add_command_option('preprocessor', self._source_filename)

        self.add_command_option('compiler', "--target " + self._target)
        self.add_command_option('compiler', "--arch " + self._arch)
        self.add_command_option('compiler', "-o")
        self.add_command_option('compiler', "{}".format(output_dir))
        self.add_command_option('compiler', "{}.p4pp".format(basepath))
        # cleanup after compiler
        if not opts.debug_info:
            self._postCmds['compiler'] = []
            self._postCmds['compiler'].append(["rm -f {}.p4pp".format(basepath)])

        # cleanup after assembler
        # self._postCmds['assembler'] = []
        # if not opts.debug_info:
        #     self._postCmds['assembler'].append(["rm -f {}.bfa".format(basepath)])

        src_filename, src_extension = os.path.splitext(self._source_filename)
        # local options
        if opts.run_post_compiler or src_extension == '.bfa':
            self.enable_commands(['assembler'])

        if opts.skip_linker:
            self._no_link = True

        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            if opts.gdb or opts.cgdb or opts.lldb:
                # XXX breaks abstraction
                old_command = self._commands['compiler']
                if opts.lldb:
                    self.add_command('compiler', 'lldb')
                    self.add_command_option('compiler', '--')
                else:
                    self.add_command('compiler', 'gdb' if opts.gdb else 'cgdb')
                    self.add_command_option('compiler', '--args')
                for arg in old_command:
                    self.add_command_option('compiler', arg)

        if opts.create_graphs or opts.archive:
            self.add_command_option('compiler', '--create-graphs')

        if opts.backward_compatible:
            self.add_command_option('compiler', '--backward-compatible')

        if opts.parser_timing_reports:
            self.add_command_option('compiler', '--parser-timing-reports')

        if opts.parser_bandwidth_opt:
            self.add_command_option('compiler', '--parser-bandwidth-opt')

        if opts.no_dead_code_elimination:
            self.add_command_option('compiler', '--no-dead-code-elimination')

        if opts.auto_init_metadata:
            self.add_command_option('compiler', '--auto-init-metadata')

        if opts.placement:
            self.add_command_option('compiler', '--placement=pragma')

        if opts.egress_intrinsic_metadata_opt:
            self.add_command_option('compiler', '--egress-intrinsic-metadata-opt')

        if opts.log_hashes:
            self.add_command_option('assembler', '--log-hashes')

        if opts.quick_phv_alloc:
            self.add_command_option('compiler', '--quick-phv-alloc')

        self.skip_compilation = []
        if opts.skip_compilation:
            self.add_command_option(
                'compiler', '--skip-compilation={}'.format(opts.skip_compilation)
            )
            self.skip_compilation = opts.skip_compilation.split(',')

        if opts.display_power_budget:
            self.add_command_option('compiler', '--display-power-budget')

        if opts.ir_to_json is not None:
            self.add_command_option('compiler', '--toJSON {}'.format(opts.ir_to_json))
            self.enable_commands(['preprocessor', 'compiler'])
            self.runVerifiers = False
        self._ir_to_json = opts.ir_to_json

        # Developer only option
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            if opts.pretty_print is not None:
                self.runVerifiers = False

        if opts.disable_egress_latency_padding:
            self.add_command_option('assembler', '--disable-egress-latency-padding')

        if opts.help_warnings:
            self.add_command_option('compiler', '--help-warnings')
            self.checkAndRunCmd('compiler')
            # no need for anything else, we printed the pragmas and we need to exit
            sys.exit(0)

        if opts.Wdisable is not None:
            self.config_warning_modifiers(opts.Wdisable, "disable")

        if opts.Werror is not None:
            self.config_warning_modifiers(opts.Werror, "error")

        self.pragmas_help = opts.help_pragmas
        if opts.help_pragmas:
            self.add_command_option('compiler', '--help-pragmas')
            self.checkAndRunCmd('compiler')
            # no need for anything else, we printed the pragmas and we need to exit
            sys.exit(0)

        if opts.verbose > 0:
            ta_logging = "table_placement:3,table_summary:1,table_dependency_graph:3,table_dependency_summary:3"
            phv_verbosity = str(2 * opts.verbose - 1)
            pa_logging = (
                "allocate_phv:"
                + phv_verbosity
                + ",allocator_base:3"
                + ",trivial_allocator:5"
                + ",greedy_allocator:5"
                + ",alias:1"
            )
            parde_verbosity = str(2 * opts.verbose - 1)
            parde_logging = (
                ",allocate_clot:"
                + parde_verbosity
                + ",clot_info:"
                + parde_verbosity
                + ",split_parser_state:"
                + parde_verbosity
                + ",allocate_parser_match_register:"
                + parde_verbosity
                + ",allocate_parser_checksum:"
                + parde_verbosity
                + ",lower_parser:"
                + parde_verbosity
                + ",decaf*:"
                + parde_verbosity
                + ",characterize_parser.h:"
                + parde_verbosity
            )
            bridge_logging = "bridged_packing:1"
            ixbar_logging = "ixbar_info:3"
            self.add_command_option(
                'compiler',
                '--verbose -T{},{},{},{},{}'.format(
                    ta_logging, pa_logging, parde_logging, bridge_logging, ixbar_logging
                ),
            )

        if opts.debug_info:
            self.add_command_option(
                'compiler', '-Tstage_advance:3>{}/stage_adv.log'.format(self._output_directory)
            )

        # re-apply user provided options to override default values
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            for option in opts.log_levels:
                self.add_command_option('compiler', "-T{}".format(option))

        # Print all used schema versions
        if opts.schema_versions:
            schema_versions_file = open(
                os.path.join(os.environ['P4C_CFG_PATH'], 'schema_versions'), 'r'
            )
            print(schema_versions_file.read().rstrip('\n'))
            schema_versions_file.close()

        if opts.num_stages_override:
            if 'assembler' in self._commandsEnabled and 'compiler' in self._commandsEnabled:
                self.add_command_option(
                    'assembler', "--num-stages-override{}".format(opts.num_stages_override)
                )
                self.add_command_option(
                    'compiler', "--num-stages-override={}".format(opts.num_stages_override)
                )

        if (
            opts.bf_rt_schema is None
            and opts.language == 'p4-16'
            and not (self._arch == 'v1model' or self._arch == 'psa' or opts.no_bf_rt_schema)
        ):
            opts.bf_rt_schema = "{}/bfrt.json".format(self._output_directory)

        if opts.bf_rt_schema is not None:
            self.add_command_option('compiler', '--bf-rt-schema {}'.format(opts.bf_rt_schema))

            if self.runVerifiers:
                self.add_command_option('bf-rt-verifier', opts.bf_rt_schema)
                if 'compiler' in self._commandsEnabled:
                    self._commandsEnabled.append('bf-rt-verifier')

        if opts.p4runtime_force_std_externs:
            self.add_command_option('compiler', '--p4runtime-force-std-externs')

        # Add conf generation options
        if opts.bf_rt_schema is not None:
            conf_type = 'BF-RT'
            self.add_command_option('p4c-gen-conf', '--bfrt-name {}'.format(opts.bf_rt_schema))
        elif opts.language == 'p4-14':
            conf_type = 'PD'
        elif opts.p4runtime_force_std_externs:
            conf_type = 'P4Runtime'
        else:
            conf_type = 'BF-RT'  # default ...
        self.add_command_option('p4c-gen-conf', '--conf-type {}'.format(conf_type))
        self.add_command_option('p4c-gen-conf', '--name {}'.format(self.program_name))
        self.add_command_option('p4c-gen-conf', '--device {}'.format(self._target))
        self.add_command_option('p4c-gen-conf', '--outputdir {}'.format(output_dir))
        self.add_command_option('p4c-gen-conf', '--p4-version {}'.format(opts.language))
        self.conf_file = self.program_name + ".conf"

        if opts.verbose > 0:
            log_scripts_dir = os.environ['P4C_BIN_DIR']
            top_src_dir = checkEnv()
            if top_src_dir:
                # dev environment
                log_scripts_dir = os.path.join(top_src_dir, 'compiler_interfaces')
            if os.path.exists(os.path.join(log_scripts_dir, 'p4c-build-logs')):
                self.add_command('summary_logging', os.path.join(log_scripts_dir, 'p4c-build-logs'))
                self._commandsEnabled.append('summary_logging')

        # Developer only options
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
            if self.runVerifiers:
                if 'assembler' in self._commandsEnabled:
                    self._commandsEnabled.append('verifier')
                if 'compiler' in self._commandsEnabled:
                    # always validate the manifest if opts.validate_manifest or self.runVerifiers:
                    self.add_command_option(
                        'manifest-verifier', "{}/manifest.json".format(output_dir)
                    )
                    self._commandsEnabled.append('manifest-verifier')

        # if we need to generate an archive, should be the last command
        if opts.archive is not None:
            self.add_command('archiver', 'tar')
            root_dir = os.path.dirname(output_dir)
            if root_dir == "":
                root_dir = "."
            if opts.archive == "__default__":
                program_name = os.path.basename(basepath)
            else:
                program_name = opts.archive
            program_dir = os.path.basename(output_dir)
            if program_dir != ".":
                self.add_command_option(
                    'archiver',
                    "-jcf {}/{}.tar.bz2 --exclude=\"*.bin\" -C {} {}".format(
                        root_dir, program_name, root_dir, program_dir
                    ),
                )
                if opts.archive_source:
                    self.add_command_option(
                        'archiver', os.path.dirname(os.path.abspath(self._source_filename))
                    )
                self._commandsEnabled.append('archiver')
            else:
                print(
                    "Please specify an output directory (using -o) to" + " generate an archive",
                    file=sys.stderr,
                )

    def parseManifest(self):
        """! Parse the manifest file and return a map of the program pipes
        If dry-run, the manifest does not exist, so we fake one to print at least
        one assembler line if needed.
        """
        manifest_filename = "{}/manifest.json".format(self._output_directory)

        if self._dry_run:
            print('parse manifest:', manifest_filename)
            self._pipes = {
                0: {
                    'context': '{}/pipe/context.json'.format(self._output_directory),
                    'resources': '{}/pipe/resources.json'.format(self._output_directory),
                    'pipe_dir': '{}/pipe'.format(self._output_directory),
                    'pipe_id': 0,
                }
            }
            return 0

        # compilation failed and there is no manifest. An error should have been printed,
        # so we simply exit here
        if not os.path.isfile(manifest_filename) or os.path.getsize(manifest_filename) == 0:
            self.exitWithError(None)

        with open(manifest_filename, "r") as json_file:
            try:
                self._manifest = json.load(json_file)
            except:
                error_msg = None
                if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
                    error_msg = (
                        "ERROR: Input file '"
                        + manifest_filename
                        + "' could not be decoded as JSON.\n"
                    )
                self.exitWithError(error_msg)
            if type(self._manifest) is not dict or "programs" not in self._manifest:
                error_msg = None
                if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER":
                    error_msg = (
                        "ERROR: Input file '"
                        + manifest_filename
                        + "' does not appear to be valid manifest JSON.\n"
                    )
                self.exitWithError(error_msg)

        self._pipes = {}
        schema_version = parse_version(self._manifest['schema_version'])
        pipe_name_label = 'pipe_name'
        if schema_version == parse_version("1.0.0"):
            pipe_name_label = 'pipe'

        programs = self._manifest['programs']
        if len(programs) > 1:
            error_msg = "{} currently supports a single program".format(self._targetName.title())
            self.exitWithError(error_msg)

        def __parseManifestBefore_2_0(prog, p4_version):
            if type(prog) is not dict or "contexts" not in prog:
                error_msg = (
                    "ERROR: Input file '"
                    + manifest_filename
                    + "' does not contain valid program contexts.\n"
                )
                self.exitWithError(error_msg)
            for ctxt in prog["contexts"]:
                pipe_id = ctxt['pipe']
                self._pipes[pipe_id] = {}
                self._pipes[pipe_id]['pipe_name'] = ctxt['pipe_name']
                self._pipes[pipe_id]['context'] = os.path.join(self._output_directory, ctxt['path'])
                if p4_version == 'p4-14':
                    self._pipes[pipe_id]['pipe_dir'] = self._output_directory
                else:
                    self._pipes[pipe_id]['pipe_dir'] = os.path.join(
                        self._output_directory, ctxt['pipe_name']
                    )
            for res in prog['p4i']:
                pipe_id = res['pipe']
                res_file = os.path.join(self._output_directory, res['path'])
                self._pipes[pipe_id]['resources'] = res_file

        def __parseManifestAfter_2_0(prog, p4_version):
            if type(prog) is not dict or "pipes" not in prog:
                error_msg = (
                    "ERROR: Input file '"
                    + manifest_filename
                    + "' does not contain a valid program.\n"
                )
            for pipe in prog["pipes"]:
                if pipe['pipe_name'] in self.skip_compilation:
                    continue
                pipe_id = int(pipe['pipe_id'])
                self._pipes[pipe_id] = {}
                self._pipes[pipe_id]['pipe_id'] = pipe_id
                self._pipes[pipe_id]['pipe_name'] = pipe['pipe_name']
                self._pipes[pipe_id]['context'] = os.path.join(
                    self._output_directory, pipe['files']['context']['path']
                )
                if p4_version == 'p4-14':
                    self._pipes[pipe_id]['pipe_dir'] = self._output_directory
                else:
                    self._pipes[pipe_id]['pipe_dir'] = os.path.join(
                        self._output_directory, pipe['pipe_name']
                    )
                for res in pipe['files']['resources']:
                    if res['type'] == "resources":
                        self._pipes[pipe_id]['resources'] = os.path.join(
                            self._output_directory, res['path']
                        )
                for graph in pipe['files']['graphs']:
                    if graph['graph_format'] == ".json" and graph['graph_type'] == 'table':
                        self._pipes[pipe_id]['graph'] = os.path.join(
                            self._output_directory, graph['path']
                        )
                for log in pipe['files']['logs']:
                    if log['log_type'] == 'phv' and log['path'].endswith('phv.json'):
                        self._pipes[pipe_id]['phv_json'] = os.path.join(
                            self._output_directory, log['path']
                        )
                    elif log['log_type'] == 'power' and log['path'].endswith('power.json'):
                        self._pipes[pipe_id]['power_json'] = os.path.join(
                            self._output_directory, log['path']
                        )

        for prog in programs:
            p4_version = prog['p4_version']
            if schema_version < parse_version("2.0.0"):
                __parseManifestBefore_2_0(prog, p4_version)
            else:
                __parseManifestAfter_2_0(prog, p4_version)

    def updateManifest(self, jsonFile, compilation_successful=True):
        """! Set the compile_command in the manifest or context.json
        @param jsonFile Path to the json file
        @param compilation_successful Updated value for compilation_succeeded attribute
        """
        if self._dry_run or not os.path.exists(jsonFile) or os.path.getsize(jsonFile) == 0:
            return

        jsonTree = None
        with open(jsonFile, "r") as json_file:
            try:
                jsonTree = json.load(json_file)
                jsonTree['compile_command'] = ' '.join(sys.argv)
                jsonTree['compilation_succeeded'] = compilation_successful
                jsonTree['compilation_time'] = str(self.compilation_time)
                jsonTree['programs'][0]['source_files']['src_root'] = os.path.dirname(
                    os.path.abspath(self._source_filename)
                )
                if self.conf_file is not None:
                    jsonTree["target_data"]['conf_file'] = self.conf_file
                for pipe in self.mau_json:
                    mau_json = {'path': self.mau_json[pipe], 'log_type': 'mau'}
                    jsonTree['programs'][0]['pipes'][pipe]['files']['logs'].append(mau_json)
                for pipe in self.metrics:
                    jsonTree['programs'][0]['pipes'][pipe]['files']['metrics'] = {
                        'path': self.metrics[pipe]
                    }
                for pipe in self.contexts:
                    jsonTree['programs'][0]['pipes'][pipe]['files']['context'] = {
                        'path': self.contexts[pipe]
                    }
            except:
                return

        if jsonTree is not None:
            with open(jsonFile, "w") as new_file:
                json.dump(jsonTree, new_file, indent=2, separators=(',', ': '))

    def exitWithError(self, error_msg):
        """! Function to be called when compilation ends in error.
        @param error_msg Error message to be emitted
        """
        try:
            manifest_json = os.path.join(self._output_directory, 'manifest.json')
            self.updateManifest(manifest_json, False)
        except:
            pass
        finally:
            if error_msg is not None:
                print(str(error_msg), file=sys.stderr)
            sys.exit(1)

    def runAssembler(self, dirname, unique_table_offset):
        """! Run an instance of the assembler on the provided directory
        @param dirname Name of the directory on which to run the assembler
        @param unique_table_offset --table-handle-offset argument value
        @return Assemblers return code
        """
        # reset all assembler options to what was passed on cmd line
        # Note that we need to make a copy of the list
        self._commands['assembler'] = list(self._saved_assembler_params)
        # lookup the directory name. For P4-16, it is the output + pipe_name.
        # This logging feature is enabled during the DEVELOPER mode
        if os.environ['P4C_BUILD_TYPE'] == "DEVELOPER" and self.verbose > 0:
            self.add_command_option('assembler', "-vvvl {}/bfas.config.log".format(dirname))
        else:
            # Disable warnings when not in DEVELOPER Mode
            self.add_command_option('assembler', "--no-warn")

        # don't generate a binary
        if self._no_link:
            self.add_command_option('assembler', "--no-bin")

        # target name
        self.add_command_option('assembler', "--target " + self._targetName)
        # prepend unique offset to table handle
        self.add_command_option('assembler', "--table-handle-offset{0}".format(unique_table_offset))

        if self._multi_parsers:
            self.add_command_option('assembler', "--multi-parsers")

        # output dir
        self.add_command_option('assembler', "-o {}".format(dirname))
        # input file
        asm_file = "{}/{}.bfa".format(dirname, self.program_name)
        asm_file_path = os.path.join(os.getcwd(), asm_file)
        if not self._dry_run and not os.path.isfile(asm_file_path):
            print("Skipping assembler, no assembly file generated", file=sys.stderr)
            return 1

        if not self._dry_run and os.path.getsize(asm_file) == 0:
            print("Skipping assembler, assembly file is empty", file=sys.stderr)
            return 1

        self.add_command_option('assembler', asm_file)
        # run
        return self.checkAndRunCmd('assembler')

    def runSummaryLogging(self, pipe):
        """! Does summary logging
        @param pipe Pipe to log
        @return Return code
        """

        def __update_log_file(filemap, filetype, filename):
            fpath = (
                os.path.join(filetype, filename)
                if self.language == 'p4-14'
                else os.path.join(pipe['pipe_name'], filetype, filename)
            )
            if os.path.exists(os.path.join(self._output_directory, fpath)):
                filemap[pipe['pipe_id']] = fpath

        try:
            self.add_command_option('summary_logging', "{}".format(pipe['context']))
            if pipe.get('resources', False):
                self.add_command_option('summary_logging', "-r {}".format(pipe['resources']))
            self.add_command_option(
                'summary_logging', "-o {}".format(os.path.join(pipe['pipe_dir'], 'logs'))
            )
            self.add_command_option('summary_logging', "--disable-phv-json")
            if pipe.get('power_json', False):
                self.add_command_option('summary_logging', "-p {}".format(pipe['power_json']))
            manifest_filename = "{}/manifest.json".format(self._output_directory)
            self.add_command_option('summary_logging', "-m {}".format(manifest_filename))
            rc = self.checkAndRunCmd('summary_logging')
            # and now remove the arguments added so that the next pipe is correct
            del self._commands['summary_logging'][1:]
            __update_log_file(self.mau_json, 'logs', 'mau.json')
            __update_log_file(self.metrics, 'logs', 'metrics.json')
            return rc
        except:
            pass
            # raise
        return 1

    def runCleaner(self):
        """! Executes cleaner process
        @return Cleaner return code
        """
        if self.debug_info:
            return 0

        # Don't forget to edit the reference list in scripts/test_p4c_driver.py file!
        filesToRemove = []
        filesToRemove.append('.dynhash.json')
        filesToRemove.append('.prim.json')
        filesToRemove.append('resources_deparser.json')

        self.add_command_option('cleaner', '-f')
        filesFound = 0
        for root, dirs, files in os.walk(self._output_directory):
            for f in files:
                for rFile in filesToRemove:
                    if f.endswith(rFile):
                        self.add_command_option('cleaner', os.path.join(root, f))
                        filesFound += 1

        if filesFound == 0:
            return 0

        return self.checkAndRunCmd('cleaner')

    def checkAndRunCmd(self, command):
        """! Checks command existance and runs the command
        @param command Command to be run
        @return Return code of the command
        """
        # this should be in the parent class!!
        cmd = self._commands[command]
        if cmd[0].find('/') != 0 and (find_bin(cmd[0]) == None):
            error_msg = "{}: command not found".format(cmd[0])
            print(str(error_msg), file=sys.stderr)
            sys.exit(100)  # environment missconfiguration.
        rc = self.runCmd(command, cmd)
        if rc != 0:
            error_msg = "failed command {}".format(command)
            print(str(error_msg), file=sys.stderr)
        # in the Python doc, a return code of None means the child did not terminate.
        # However, we seem to get a None even when the subprocess actually terminated ...
        # sometimes successfuly. We still consider a return code of None to be a failure as
        # we want to see that 0 on success!
        if rc is None:
            rc = 1
        return rc

    def checkVersionTargetArch(self, target, language, arch):
        """! Set the architecture and other attributes based on set values
        @param target Target device
        @param language P4 language standard
        @param arch Target architecture
        """
        if language == "p4-14" and arch == 'default':
            self._arch = "v1model"
            self.backend = target + '-' + 'v1model'
        elif language == "p4-16" and arch == 'default':
            match = re.match('tofino([0-9]?)', target)
            rev = match.group(1) or ''
            self._arch = 't' + rev + 'na'
            self.backend = target + '-' + self._arch

    def aggregate_deparser_resources_json(self, pipe):
        """! This method joins together all files from the p4c-barefoot and
        assembler generator.
        @param pipe Object with available pipes
        """
        # Prepare path for output files
        log_dir = os.path.join(self._output_directory, pipe["pipe_name"], "logs")
        deparser_file = os.path.join(log_dir, "resources_deparser.json")
        if "resources" in pipe.keys():
            resources_file = pipe["resources"]
        else:
            # No resources generated, nothing to add
            return

        if not (os.path.exists(resources_file)) or not (os.path.exists(deparser_file)):
            # Any of required files doesn't exist
            return

        # So far so good, open both files and add the bf-asm file to the resources
        # file under deparser node
        resources_json = open(resources_file, "r+")
        deparser_json = open(deparser_file, "r")

        # Append the deparer node to the output
        deparser_data = json.load(deparser_json)
        resources_data = json.load(resources_json)
        resources_data["resources"]["deparser"] = deparser_data

        # Dump the node to the output - don't forget to reset the file
        resources_json.close()
        resources_json = open(resources_file, "w")
        json.dump(resources_data, resources_json, indent=2)
        resources_json.close()

    def run(self):
        """! Override the parent run, in order to insert manifest parsing.
        @return Return code
        """
        run_assembler = 'assembler' in self._commandsEnabled
        run_archiver = 'archiver' in self._commandsEnabled
        run_compiler = 'compiler' in self._commandsEnabled
        run_verifier = 'verifier' in self._commandsEnabled
        run_manifest_verifier = 'manifest-verifier' in self._commandsEnabled
        run_summary_logs = 'summary_logging' in self._commandsEnabled
        run_p4c_gen_conf = 'p4c-gen-conf' in self._commandsEnabled
        run_cleaner = 'cleaner' in self._commandsEnabled

        # run the preprocessor, compiler, and verifiers (manifest, context schema, and bf-rt)
        self.disable_commands(
            ['assembler', 'archiver', 'verifier', 'summary_logging', 'p4c-gen-conf', 'cleaner']
        )

        start_t = time.time()
        rc = BackendDriver.run(self)
        self.compilation_time = time.time() - start_t

        # Error codes defined in p4c-barefoot.cpp:main
        if rc is None:
            rc = 1
        if rc > 1 or rc < 0:
            # Invocation or program error. Should try to recover as much as we can
            try:
                self.parseManifest()
                for pipe in self._pipes.values():
                    if run_summary_logs and pipe.get('context', False):  # context.json is required
                        # ignore the return code -- we may have failed generating some logs.
                        # update manifest to export compilation time before runSummaryLogging is executed
                        self.updateManifest(
                            os.path.join(self._output_directory, 'manifest.json'), False
                        )
                        self.runSummaryLogging(pipe)
                self.updateManifest(os.path.join(self._output_directory, 'manifest.json'), False)
                if run_cleaner:
                    self.runCleaner()
                if run_archiver:
                    self.checkAndRunCmd('archiver')
            finally:
                return rc

        # ir_to_json exits early, serializing only the IR
        # print pragmas also needs to exit early, it's just like help
        if self._ir_to_json is not None or self.pragmas_help:
            return rc

        # we ran the compiler, now we need to parse the manifest and run the assembler
        # for each P4-16 pipe
        rc_bfa = 0  # accumulate assembler errors
        if run_assembler:
            self.parseManifest()
            # We need to make a copy of the list to get a copy of any additional parameters
            # that were added on the command line (-Xassembler)
            self._saved_assembler_params = list(self._commands['assembler'])
            unique_table_offset = 0
            for pipe in self._pipes.values():

                if 'pipe_name' in pipe and pipe['pipe_name'] in self.skip_compilation:
                    continue

                start_t = time.time()
                rc_bfa += self.runAssembler(pipe['pipe_dir'], unique_table_offset)
                self.compilation_time += time.time() - start_t
                unique_table_offset += 1

                # We always need a  context.json -- TODO: need to make sure it is generated
                pipeName = 'pipe' if self._dry_run else pipe['pipe_name']
                context = (
                    'context.json'
                    if self.language == 'p4-14'
                    else os.path.join(pipeName, 'context.json')
                )
                pipe['context'] = os.path.join(self._output_directory, context)
                self.contexts[pipe['pipe_id']] = context
                # Although the context.json schema has an optional compile_command and
                # we could add it here, it is a potential performance penalty to re-write
                # a large context.json file. So we don't!

                # Add resources from deparser
                if self._dry_run:
                    print(
                        "Skipping aggregation of resources_deparser.json with resources.json, no file was generated"
                    )
                else:
                    self.aggregate_deparser_resources_json(pipe)

                rc_ver = 0
                if run_verifier:
                    # A map of file key and verifier option
                    toBeVerified = {
                        'context': 'c',
                        'graph': 'd',
                        'resources': 'r',
                        'phv_json': 'p',
                        'power_json': 'w',
                        'source': 's',
                        # add new option
                    }
                    # Clear verifier options
                    del self._commands['verifier'][1:]
                    for k in sorted(toBeVerified):
                        if pipe.get(k, False) and os.path.exists(pipe[k]):
                            self.add_command_option(
                                'verifier', "-{} {}".format(toBeVerified[k], pipe[k])
                            )
                    rc_ver = self.checkAndRunCmd('verifier')

                if run_summary_logs and rc_ver == 0:
                    if pipe.get('context', False):  # context.json is required
                        # update manifest to export compilation time before runSummaryLogging is executed
                        self.updateManifest(
                            os.path.join(self._output_directory, 'manifest.json'), False
                        )
                        rc += self.runSummaryLogging(pipe)

                rc += rc_ver

                # TODO: the assembler failed: should we assemble the other pipes? Now we do.

        success = (rc + rc_bfa) == 0
        self.updateManifest(os.path.join(self._output_directory, 'manifest.json'), success)
        if success and run_p4c_gen_conf:
            if self._dry_run:
                pipeNames = ['pipe']
            else:
                pipeNames = [p['pipe_name'] for p in self._pipes.values()]
            self.add_command_option('p4c-gen-conf', '--pipe {}'.format(' '.join(pipeNames)))
            rc += self.checkAndRunCmd('p4c-gen-conf')

        # Chech manifest only after all changes have been made to it
        rmc = 0
        if run_manifest_verifier:
            rmc = self.checkAndRunCmd('manifest-verifier')

        if rmc != 0:
            print("Manifest validation failed")
            return rmc

        # Cleanup temp files
        if run_cleaner:
            self.runCleaner()

        # run the archiver if one has been set, regardless whether the
        # execution was successful or not
        if run_archiver:
            rc += self.checkAndRunCmd('archiver')

        # We've successfully reached this point, but the compilation may have failed
        return rc + rc_bfa
